#!/usr/bin/python
# -*- coding: UTF8 -*-

""" docstring """

__author__      = "Hao FengGe (xxhfg@163.com)"
__version__     = "@Revision: 0.1 $"
__date__        = "@Update  : 2011/06/09 17:22:07 $"
__copyright__   = "Copyright (c) 2010 Hao FengGe"
__license__     = "Python"

PROJECT_HOME='d:/webroot/shuyou/'
#PROJECT_HOME='/home/administrator/workspace/shuyou/'

import sys
sys.path.append(PROJECT_HOME) #先要把自己的项目目录加入path
from django.core.management import setup_environ #这是重头戏，全靠它了
from shuyou import settings #介绍自已人
setup_environ(settings) #安排自己人
#干活去吧
# print settings.DATABASES
from django import template
from book import models
import urllib
import random
import Queue  
import threading  
import time  
import re
import socket
import httplib2
from datetime import datetime
from django.db import transaction
'''
'''
class WorkManager(object):  
    def __init__(self, arg_list=[],thread_num=2):  
        self.work_queue = Queue.Queue()  
        self.threads = []  
        self.__init_work_queue(arg_list)  
        self.__init_thread_pool(thread_num)  

    """ 
    初始化线程 
    """  
    def __init_thread_pool(self,thread_num):  
        for i in range(thread_num):  
            self.threads.append(Work(self.work_queue))  

    """ 
    初始化工作队列 
    """  
    def __init_work_queue(self, arg_list):  
        for arg in arg_list:
            self.add_job(do_job, arg)  


    """ 
    添加一项工作入队 
    """  
    def add_job(self, func, *args):  
        self.work_queue.put((func, list(args)))#任务入队，Queue内部实现了同步机制  

    """ 
    等待所有线程运行完毕 
    """     
    def wait_allcomplete(self):  
        for item in self.threads:  
            if item.isAlive():item.join()  

class Work(threading.Thread):  
    def __init__(self, work_queue):  
        threading.Thread.__init__(self)  
        self.work_queue = work_queue  
        self.start()  

    def run(self):  
        #死循环，从而让创建的线程在一定条件下关闭退出  
        while True:  
            try:  
                do, args = self.work_queue.get(block=False)#任务异步出队，Queue内部实现了同步机制  
                do(args)  
                self.work_queue.task_done()#通知系统任务完成  
            except:  
                break  

#具体要做的任务  
def do_job(args):  
    global g_mutex
    # global contents, novels
    contents = []
    break_job = False

    for Publisher in args:
        url=template.Template(Publisher.update_page)
        num = 0
        for i in range(1, 101):
            if break_job:
                break

            time.sleep(2)
            # print u"正在处理%s, 第%3d页" % (Publisher.name, i)
            c=template.Context({'page':i})
            list_url=url.render(c)

            opener = urllib.FancyURLopener({})      #不使用代理
            #www.my-proxy.com 需要下面这个Cookie才能正常抓取
            opener.addheaders = [
                ('User-agent','Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)'),
                ('Cookie','permission=1'), 
            ]
            t=time.time()
            if (list_url.find("?")==-1):
                list_url=list_url+'?rnd='+str(random.random())
            else:
                list_url=list_url+'&rnd='+str(random.random())

            # try:
            #     f = opener.open(list_url)
            #     print f.info().headers
            #     html = f.read()        
            # except:
            #     html = ''

            try:
                # h = httplib2.Http('cache')
                h = httplib2.Http()
                response, html = h.request(list_url)
                if response.status <> 200:
                    html = ''

                html = mdcode(html)
                # last_modified = datetime.strptime(response['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')
                # last_time = time.mktime(last_modified.timetuple())
                # if Publisher.last_modified and (Publisher.last_modified >=  last_time):
                    # html = ''
                # else:
                    # Publisher.last_modified = last_time
            except:
                html = ''

            if html <> '':
                pattern = re.compile(Publisher.update_pattern, re.IGNORECASE)
                m= re.finditer(pattern, html)
                if m:
                    ml = list(m)
                    # print "共匹配%4d条记录！" % len(ml)
                    for a in ml:
                        c = a.groupdict()
                        if 'novel_code' in c:
                            pass
                        else:
                            c['novel_code'] = ''
                        if 'content_code' in c:
                            pass
                        else:
                            c['content_code'] = ''
                        if 'update_time' in c:
                            pass
                        else:
                            c['update_time'] = ''
                        if 'novel_name' in c:
                            pass
                        else:
                            c['novel_name'] = ''
                        if 'content_title' in c:
                            pass
                        else:
                            c['content_title'] = ''
                        c['last_update'] = c['novel_name'] + '_' + c['content_title'] + '_' + c['update_time']
                        if unicode(c['last_update'], 'UTF8') == Publisher.last_update:
                            break_job = True
                            break

                        num += 1
                        c['index'] = num
                        c['publisher'] = Publisher
                        c['publisher_name'] = Publisher.name
                        contents.append(c)
                        # for k in a.groupdict():
                            # print k, a.group(k)

    
            # filename = Publisher.name + '_' + str(i) + '_' + ".html"
            # with open(filename, 'w') as file:
            # file.write(html)

    g_mutex.acquire()
    do_contents(contents)
    g_mutex.release()

def mdcode( str, encoding='utf-8' ):
    if isinstance(str, unicode):
        return str.encode(encoding)

    for c in ('utf-8', 'gbk', 'gb2312','gb18030','utf-16'):
        try:
            if encoding == 'unicode':
                return str.decode(c)
            else:
                return str.decode(c).encode( encoding )
        except:
            pass

    raise Exception('Unknown charset')

def do_author(content, authors, max_author_id):
    # authors = models.Author.objects.all()
    for author in authors:
        if mdcode(author.name) == mdcode(content['author_name']):
            content['author_id'] = author.id
            break

    if 'author_id' in content:
        pass
    else:#新作者
        max_author_id = max_author_id + 1
        content['author_id'] = max_author_id
        author = models.Author(name=content['author_name'], id=content['author_id'])
        author.save()
        
    return max_author_id

def do_publisher(content):
    if ('index' in content) and (content['index'] == 1):
        publisher = content['publisher']
        publisher.last_update = content['last_update']
        publisher.save()

    return content['publisher']

def do_novel(content, novels, max_novel_id):
    # novels = models.Novel.objects.all()
    for novel in novels:
        if mdcode(novel.alias_name) == mdcode(content['alias_name']):
            content['novel_id'] = novel.id
            break
    
    if 'novel_id' in content: 
        pass
    else:#新作品
        max_novel_id = max_novel_id + 1
        content['novel_id'] = max_novel_id

        novel = models.Novel(name=content['novel_name'], id=content['novel_id'], alias_name=content['alias_name'], author_id=content['author_id'])
        novel.save()

        content['novelpage']=do_novel_page(content)

    return max_novel_id


def do_novel_page(content):
    publisher = content['publisher']
    url_t = template.Template(publisher.novel_page)
    c = template.Context({'novel_code':content['novel_code']})
    url = url_t.render(c)
    content['novel_page'] = url
    novel_page = models.NovelPage(novel_id=content['novel_id'], publisher_id=publisher.id, novel_code=content['novel_code'], novel_page=url)
    novel_page.save()

    return novel_page.id

def do_content_page(content):
    publisher = content['publisher']
    url_t = template.Template(publisher.content_page)

    c = template.Context({'vip':content['vip'], 'novel_code':content['novel_code'], 'content_code':content['content_code']})
    url = url_t.render(c)
    content['content_page'] = url
    # print url
    content_page = models.ContentPage(title=content['content_title'], novel_id=content['novel_id'], publisher_id=content['publisher'].id, novelpage_id=0, novel_code=content['novel_code'], content_code=content['content_code'], content_page=content['content_page'], update_time=content['append_time'])
    content_page.save()

    return content_page.id

@transaction.commit_on_success
def do_contents(contents):
    valids = {}
        
    try:
        authors = models.Author.objects.all()
        novels = models.Novel.objects.all()
        max_author_id = len(authors)
        max_novel_id = len(novels)
        for content in contents:
          valid_content = content['novel_name'] + '_' + content['content_title']
          if  (valid_content in valids):
              if content['index'] == 1:
                  raise Exception('网站更新过快！')
              else:
                  pass
          else:

            if ('vip' in content):
                if (content['vip'] == None):
                    content['vip'] = ''
            else:
                content['vip'] = ''

            if ('novel_code' in content):
                if (content['novel_code'] == None):
                    content['novel_code'] = ''
            else:
                content['novel_code'] = ''

            if ('content_code' in content):
                if (content['content_code'] == None):
                    content['content_code'] = ''
            else:
                content['content_code'] = ''

            # if ('year' in content):
                # if (len(content['year']) == 2):
                    # content['year'] = '20' + content['year']
            # else:
                # content['year'] = time.strftime("%Y", time.localtime())
            # if 'time' in content:
                # pass
            # else:
                # content['time'] = ''

            # content['update_time'] = content['year'] + '-' + content['month'] + '-' + content['day'] + ' ' + content['time']
            content['append_time'] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

            content['alias_name'] = content['novel_name'] + '_' + content['author_name']
            # content['last_update'] = content['novel_code'] + '_' + content['content_code'] + '_' + content['update_time']

            # print content['publisher_name'] ,  content['alias_name']
            do_publisher(content)
            max_author_id=do_author(content, authors, max_author_id)
            max_novel_id=do_novel(content, novels, max_novel_id)
            do_content_page(content)
            valids[valid_content] = content['index']

    finally:
        pass

if __name__ == '__main__':  
    from singleinstance import * 
    print settings.DATABASES
#    appInstance = ApplicationInstance( '/tmp/shuyou_spider.pid' )

    g_mutex = threading.Lock()
    socket.setdefaulttimeout(10)

    models.Author.objects.all().delete()
    models.Novel.objects.all().delete()
    models.NovelPage.objects.all().delete()
    models.ContentPage.objects.all().delete()

    # contents = []
    publishers=models.Publisher.objects.filter(actived=True)
    start = time.time()  
    work_manager =  WorkManager(publishers, 2)#或者work_manager =  WorkManager(10000, 20)  
    work_manager.wait_allcomplete()  
    end = time.time()  

    # do_contents(contents)

    # print "采集完毕！"
#    appInstance.exitApplication()

